{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import numpy as np\n",
    "from torch.nn import functional as F\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "from skimage.transform import resize\n",
    "from collections import deque\n",
    "from IPython.display import clear_output, display\n",
    "import torch.multiprocessing as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00966575,  0.04722485, -0.01757739,  0.03419526])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#env = gym.make(\"Pong-v0\")\n",
    "env = gym.make(\"CartPole-v1\")\n",
    "env.reset()\n",
    "#env.unwrapped.get_action_meanings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ActorCritic, self).__init__()\n",
    "        self.l1 = nn.Linear(4,25)\n",
    "        self.l2 = nn.Linear(25,50)\n",
    "        self.actor_lin1 = nn.Linear(50,2)\n",
    "        self.l3 = nn.Linear(50,25)\n",
    "        self.critic_lin1 = nn.Linear(25,1)\n",
    "\n",
    "    def forward(self,x):\n",
    "        x = F.normalize(x,dim=0)\n",
    "        y = F.relu(self.l1(x))\n",
    "        y = F.relu(self.l2(y))\n",
    "        actor = F.log_softmax(self.actor_lin1(y),dim=0)\n",
    "        c = F.relu(self.l3(y.detach()))\n",
    "        critic = torch.tanh(self.critic_lin1(c))\n",
    "        return actor, critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(worker_model):\n",
    "    test_env = gym.make(\"CartPole-v1\")\n",
    "    test_env.reset()\n",
    "    maxrun = 0\n",
    "    done = False\n",
    "    env.reset()\n",
    "    raw_state = np.array(test_env.env.state)\n",
    "    state = torch.from_numpy(raw_state).float()\n",
    "    while(done==False):\n",
    "        #env.render('human')\n",
    "        policy, value = worker_model(state)\n",
    "        #sample action\n",
    "        action = torch.distributions.Categorical(logits=policy.view(-1)).sample().detach().numpy()\n",
    "        state_, reward, done, lives = test_env.step(action)\n",
    "        #print(value,reward)\n",
    "        state = torch.from_numpy(state_).float()\n",
    "        maxrun += 1\n",
    "    test_env.close()\n",
    "    return maxrun\n",
    "\n",
    "def update_params(worker_opt,values,logprobs,rewards,G,clc=0.1,gamma=0.95):\n",
    "        rewards = torch.Tensor(rewards).flip(dims=(0,)).view(-1)\n",
    "        logprobs = torch.stack(logprobs).flip(dims=(0,)).view(-1) #to Tensor and reverse\n",
    "        values = torch.stack(values).flip(dims=(0,)).view(-1) #to Tensor and reverse\n",
    "        Returns = []\n",
    "        ret_ = G\n",
    "        for r in range(rewards.shape[0]):\n",
    "            ret_ = rewards[r] + gamma * ret_\n",
    "            Returns.append(ret_)\n",
    "        Returns = torch.stack(Returns).view(-1)\n",
    "        Returns = F.normalize(Returns,dim=0)\n",
    "        actor_loss = -1*logprobs * (Returns - values.detach())\n",
    "        critic_loss = torch.pow(values - Returns,2)\n",
    "        loss = actor_loss.sum() + clc*critic_loss.sum()\n",
    "        loss.backward()\n",
    "        worker_opt.step()\n",
    "        return actor_loss, critic_loss\n",
    "        \n",
    "def run_episode(worker_env, worker_model, N_steps=10):\n",
    "    raw_state = np.array(worker_env.env.state)\n",
    "    state = torch.from_numpy(raw_state).float()\n",
    "    values, logprobs, rewards = [],[],[]\n",
    "    done = False\n",
    "    j=0\n",
    "    G=torch.Tensor([0])\n",
    "    while (j < N_steps and done == False):\n",
    "        j+=1\n",
    "        #run actor critic model\n",
    "        policy, value = worker_model(state)\n",
    "        values.append(value)\n",
    "        #sample action\n",
    "        logits = policy.view(-1)\n",
    "        action_dist = torch.distributions.Categorical(logits=logits)\n",
    "        action = action_dist.sample()\n",
    "        logprob_ = policy.view(-1)[action]\n",
    "        logprobs.append(logprob_)\n",
    "        state_, _, done, info = worker_env.step(action.detach().numpy())\n",
    "        #reward = reward * 10\n",
    "        state = torch.from_numpy(state_).float()\n",
    "        if done:\n",
    "            reward = -10\n",
    "            worker_env.reset()\n",
    "        else:\n",
    "            reward = 1.0\n",
    "            #_,value = worker_model(state)\n",
    "            G = value.detach()\n",
    "        rewards.append(reward)\n",
    "    return values, logprobs, rewards, G\n",
    "\n",
    "def worker(t, worker_model, counter, params,eplens): #q is mp Queue\n",
    "    start_time = time.time()\n",
    "    print(\"In process {}\".format(t,))\n",
    "    #play n steps of the game, store rewards\n",
    "    worker_env = gym.make(\"CartPole-v1\")\n",
    "    worker_env.reset()\n",
    "    worker_opt = optim.Adam(lr=1e-4,params=worker_model.parameters())\n",
    "    worker_opt.zero_grad()\n",
    "    maxrun = 1\n",
    "    for i in range(params['epochs']):\n",
    "        worker_opt.zero_grad()\n",
    "        #stores\n",
    "        values, logprobs, rewards, G = run_episode(worker_env,worker_model, params['n_steps'])\n",
    "        actor_loss,critic_loss = update_params(worker_opt,values,logprobs,rewards,G)\n",
    "        counter.value = counter.value + 1\n",
    "        if i % 50 == 0:\n",
    "            eplen = evaluate(worker_model)\n",
    "            eplens.put(eplen)\n",
    "            print(\"Process: {} Maxrun: {} ALoss: {} CLoss: {}\".format(t,eplen, \\\n",
    "                      actor_loss.detach().mean().numpy(),critic_loss.detach().mean().numpy()))\n",
    "        if time.time() - start_time > 45:\n",
    "            print(\"Done 45 seconds\")\n",
    "            break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"%%time\\nTestModel = ActorCritic()\\nworker_opt = optim.Adam(lr=1e-4,params=TestModel.parameters())\\nq2 = mp.Value('i',0)\\nparams = {\\n    'epochs':5,\\n    'n_steps':5,\\n    'n_workers':1,\\n}\\nAC_step(0,TestModel,q2,params)\""
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''%%time\n",
    "TestModel = ActorCritic()\n",
    "worker_opt = optim.Adam(lr=1e-4,params=TestModel.parameters())\n",
    "q2 = mp.Value('i',0)\n",
    "params = {\n",
    "    'epochs':5,\n",
    "    'n_steps':5,\n",
    "    'n_workers':1,\n",
    "}\n",
    "AC_step(0,TestModel,q2,params)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <span style=\"color:red;\">Train</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In process 0\n",
      "In process 1\n",
      "In process 2\n",
      "In process 4\n",
      "In process 3\n",
      "In process 5\n",
      "In process 6\n",
      "Process: 4 Maxrun: 19 ALoss: 0.10394370555877686 CLoss: 0.0462668351829052\n",
      "Process: 1 Maxrun: 17 ALoss: -0.24389143288135529 CLoss: 0.1747657060623169\n",
      "Process: 0 Maxrun: 17 ALoss: -0.2620689868927002 CLoss: 0.19557997584342957\n",
      "Process: 2 Maxrun: 16 ALoss: 0.10148320347070694 CLoss: 0.042914845049381256\n",
      "Process: 6 Maxrun: 14 ALoss: -0.2637297213077545 CLoss: 0.196259543299675\n",
      "Process: 5 Maxrun: 17 ALoss: -0.24296186864376068 CLoss: 0.17444220185279846\n",
      "Process: 3 Maxrun: 18 ALoss: 0.10500303655862808 CLoss: 0.047194380313158035\n",
      "Process: 3 Maxrun: 21 ALoss: -0.9360821843147278 CLoss: 1.301700234413147\n",
      "Process: 0 Maxrun: 32 ALoss: 0.10427164286375046 CLoss: 0.039491038769483566\n",
      "Process: 6 Maxrun: 11 ALoss: 0.10482797771692276 CLoss: 0.03987312316894531\n",
      "Process: 1 Maxrun: 15 ALoss: 0.1030658483505249 CLoss: 0.04130054637789726\n",
      "Process: 4 Maxrun: 18 ALoss: 0.1034662276506424 CLoss: 0.04000253602862358\n",
      "Process: 2 Maxrun: 19 ALoss: 0.09962769597768784 CLoss: 0.039854150265455246\n",
      "Process: 5 Maxrun: 25 ALoss: 0.10520058870315552 CLoss: 0.04079705849289894\n",
      "Process: 3 Maxrun: 15 ALoss: 0.10797350853681564 CLoss: 0.04103069379925728\n",
      "Process: 6 Maxrun: 29 ALoss: -0.7982884645462036 CLoss: 1.2815704345703125\n",
      "Process: 0 Maxrun: 20 ALoss: 0.11228697746992111 CLoss: 0.04238035902380943\n",
      "Process: 1 Maxrun: 20 ALoss: 0.1027890220284462 CLoss: 0.03855472803115845\n",
      "Process: 2 Maxrun: 31 ALoss: 0.10166426748037338 CLoss: 0.04092404991388321\n",
      "Process: 4 Maxrun: 36 ALoss: -0.5176525115966797 CLoss: 0.5035301446914673\n",
      "Process: 5 Maxrun: 41 ALoss: -0.35498982667922974 CLoss: 0.22551338374614716\n",
      "Process: 3 Maxrun: 11 ALoss: 0.08853448927402496 CLoss: 0.03491156920790672\n",
      "Process: 6 Maxrun: 20 ALoss: 0.10038068145513535 CLoss: 0.03792198374867439\n",
      "Process: 0 Maxrun: 16 ALoss: 0.08723550289869308 CLoss: 0.0339912548661232\n",
      "Process: 4 Maxrun: 12 ALoss: 0.08343344181776047 CLoss: 0.034173738211393356\n",
      "Process: 1 Maxrun: 30 ALoss: 0.08793162554502487 CLoss: 0.0347127765417099\n",
      "Process: 2 Maxrun: 38 ALoss: 0.08879053592681885 CLoss: 0.03696561977267265\n",
      "Process: 5 Maxrun: 31 ALoss: -0.5215579867362976 CLoss: 0.4276050925254822\n",
      "Process: 0 Maxrun: 19 ALoss: -0.4158153533935547 CLoss: 0.4547621011734009\n",
      "Process: 3 Maxrun: 22 ALoss: 0.07518894970417023 CLoss: 0.029612088575959206\n",
      "Process: 6 Maxrun: 37 ALoss: -0.3473924994468689 CLoss: 0.29655721783638\n",
      "Process: 1 Maxrun: 25 ALoss: -0.2995239794254303 CLoss: 0.2285233587026596\n",
      "Process: 2 Maxrun: 22 ALoss: 0.0744309201836586 CLoss: 0.03064778447151184\n",
      "Process: 4 Maxrun: 29 ALoss: 0.0742609053850174 CLoss: 0.030723348259925842\n",
      "Process: 5 Maxrun: 21 ALoss: 0.08169611543416977 CLoss: 0.03365490585565567\n",
      "Process: 0 Maxrun: 45 ALoss: 0.05432845279574394 CLoss: 0.025632983073592186\n",
      "Process: 6 Maxrun: 35 ALoss: -0.2752382159233093 CLoss: 0.25413304567337036\n",
      "Process: 4 Maxrun: 21 ALoss: 0.07093539834022522 CLoss: 0.02912277542054653\n",
      "Process: 1 Maxrun: 20 ALoss: -0.6197646856307983 CLoss: 0.7977787256240845\n",
      "Process: 2 Maxrun: 11 ALoss: 0.08388515561819077 CLoss: 0.03163054212927818\n",
      "Process: 5 Maxrun: 18 ALoss: -0.516483724117279 CLoss: 0.4649477005004883\n",
      "Process: 3 Maxrun: 71 ALoss: 0.06602819263935089 CLoss: 0.0252191424369812\n",
      "Process: 4 Maxrun: 15 ALoss: -0.4344848096370697 CLoss: 0.330188512802124\n",
      "Process: 0 Maxrun: 24 ALoss: -0.8543039560317993 CLoss: 0.8371411561965942\n",
      "Process: 6 Maxrun: 21 ALoss: 0.04820315167307854 CLoss: 0.022148791700601578\n",
      "Process: 1 Maxrun: 14 ALoss: -0.5288543105125427 CLoss: 0.6234686970710754\n",
      "Process: 3 Maxrun: 35 ALoss: -0.49630239605903625 CLoss: 0.8463389873504639\n",
      "Process: 2 Maxrun: 19 ALoss: 0.052455514669418335 CLoss: 0.02464030683040619\n",
      "Process: 5 Maxrun: 133 ALoss: -0.3744720220565796 CLoss: 0.36805641651153564\n",
      "Process: 3 Maxrun: 26 ALoss: -0.5714595913887024 CLoss: 1.4230682849884033\n",
      "Process: 4 Maxrun: 54 ALoss: 0.0595051646232605 CLoss: 0.02405857853591442\n",
      "Process: 0 Maxrun: 71 ALoss: 0.04030675068497658 CLoss: 0.023668214678764343\n",
      "Process: 1 Maxrun: 48 ALoss: 0.0627695843577385 CLoss: 0.02572663128376007\n",
      "Process: 6 Maxrun: 67 ALoss: 0.041426341980695724 CLoss: 0.021301476284861565\n",
      "Process: 2 Maxrun: 106 ALoss: 0.049469687044620514 CLoss: 0.026237890124320984\n",
      "Process: 5 Maxrun: 38 ALoss: 0.03395068645477295 CLoss: 0.0207685474306345\n",
      "Process: 3 Maxrun: 97 ALoss: 0.01994975283741951 CLoss: 0.01979558914899826\n",
      "Process: 0 Maxrun: 42 ALoss: 0.049606021493673325 CLoss: 0.02551424317061901\n",
      "Process: 4 Maxrun: 63 ALoss: 0.04139874503016472 CLoss: 0.02194703184068203\n",
      "Process: 1 Maxrun: 31 ALoss: 0.035165462642908096 CLoss: 0.02143499068915844\n",
      "Process: 2 Maxrun: 18 ALoss: 0.024114230647683144 CLoss: 0.01945941522717476\n",
      "Process: 6 Maxrun: 38 ALoss: 0.032991763204336166 CLoss: 0.01900167018175125\n",
      "Process: 5 Maxrun: 22 ALoss: -0.6432814002037048 CLoss: 0.5185571908950806\n",
      "Process: 3 Maxrun: 35 ALoss: -0.5769088268280029 CLoss: 0.4133157432079315\n",
      "Process: 0 Maxrun: 59 ALoss: 0.03750382736325264 CLoss: 0.017679253593087196\n",
      "Process: 4 Maxrun: 18 ALoss: 0.04126277193427086 CLoss: 0.019880477339029312\n",
      "Process: 6 Maxrun: 12 ALoss: 0.056822486221790314 CLoss: 0.02192029543220997\n",
      "Process: 2 Maxrun: 43 ALoss: 0.015791380777955055 CLoss: 0.01883375644683838\n",
      "Process: 5 Maxrun: 33 ALoss: 0.025542516261339188 CLoss: 0.018598048016428947\n",
      "Process: 1 Maxrun: 105 ALoss: -9.689555008662865e-05 CLoss: 0.01886540651321411\n",
      "Process: 3 Maxrun: 37 ALoss: 0.00918386410921812 CLoss: 0.019085759297013283\n",
      "Process: 4 Maxrun: 65 ALoss: 0.02384822629392147 CLoss: 0.01758677512407303\n",
      "Process: 6 Maxrun: 91 ALoss: -0.285466730594635 CLoss: 1.536390781402588\n",
      "Process: 0 Maxrun: 87 ALoss: 0.026841452345252037 CLoss: 0.01878270134329796\n",
      "Process: 2 Maxrun: 66 ALoss: 0.03322773054242134 CLoss: 0.01869354024529457\n",
      "Process: 1 Maxrun: 96 ALoss: -0.4237697720527649 CLoss: 0.4189886152744293\n",
      "Process: 5 Maxrun: 128 ALoss: 0.04168293997645378 CLoss: 0.017659995704889297\n",
      "Process: 3 Maxrun: 143 ALoss: 0.002675666008144617 CLoss: 0.016628358513116837\n",
      "Process: 4 Maxrun: 121 ALoss: 0.007253514137119055 CLoss: 0.017648374661803246\n",
      "Process: 2 Maxrun: 111 ALoss: 0.03405017778277397 CLoss: 0.017211727797985077\n",
      "Process: 6 Maxrun: 122 ALoss: 0.006598630454391241 CLoss: 0.018070384860038757\n",
      "Process: 0 Maxrun: 179 ALoss: 0.014551976695656776 CLoss: 0.018605036661028862\n",
      "Process: 5 Maxrun: 58 ALoss: -0.014590735547244549 CLoss: 0.01594204641878605\n",
      "Process: 1 Maxrun: 130 ALoss: -0.01161225326359272 CLoss: 0.016272345557808876\n",
      "Process: 3 Maxrun: 102 ALoss: 0.030833899974822998 CLoss: 0.01853989064693451\n",
      "Process: 6 Maxrun: 112 ALoss: 0.03808310255408287 CLoss: 0.018626362085342407\n",
      "Process: 2 Maxrun: 107 ALoss: 0.03990612551569939 CLoss: 0.018989823758602142\n",
      "Process: 1 Maxrun: 127 ALoss: 0.020776625722646713 CLoss: 0.017932729795575142\n",
      "Process: 4 Maxrun: 251 ALoss: 0.052682459354400635 CLoss: 0.017683563753962517\n",
      "Process: 0 Maxrun: 301 ALoss: 0.012994383461773396 CLoss: 0.01681806705892086\n",
      "Process: 5 Maxrun: 286 ALoss: 0.022319896146655083 CLoss: 0.01712161675095558\n",
      "Process: 3 Maxrun: 131 ALoss: 0.01189428847283125 CLoss: 0.017136691138148308\n",
      "Process: 6 Maxrun: 164 ALoss: 0.03880585730075836 CLoss: 0.018322082236409187\n",
      "Process: 4 Maxrun: 38 ALoss: 0.015449387021362782 CLoss: 0.016985688358545303\n",
      "Process: 1 Maxrun: 64 ALoss: 0.008412322960793972 CLoss: 0.016852473840117455\n",
      "Process: 2 Maxrun: 241 ALoss: 0.02662491239607334 CLoss: 0.01733466051518917\n",
      "Process: 0 Maxrun: 131 ALoss: -0.014969785697758198 CLoss: 0.015519638545811176\n",
      "Process: 5 Maxrun: 129 ALoss: 0.008015576750040054 CLoss: 0.017029060050845146\n",
      "Process: 3 Maxrun: 198 ALoss: -0.41460809111595154 CLoss: 0.33327093720436096\n",
      "Process: 6 Maxrun: 224 ALoss: -0.039873819798231125 CLoss: 0.01725766621530056\n",
      "Process: 4 Maxrun: 222 ALoss: 0.01683659665286541 CLoss: 0.017883313819766045\n",
      "Process: 1 Maxrun: 214 ALoss: -0.009151837788522243 CLoss: 0.01780521497130394\n",
      "Process: 2 Maxrun: 162 ALoss: 0.0019099537748843431 CLoss: 0.0173183623701334\n",
      "Process: 0 Maxrun: 60 ALoss: 0.019956469535827637 CLoss: 0.019277526065707207\n",
      "Process: 5 Maxrun: 73 ALoss: 0.021820394322276115 CLoss: 0.01909489929676056\n",
      "Process: 3 Maxrun: 110 ALoss: 0.03992505371570587 CLoss: 0.016659865155816078\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process: 6 Maxrun: 83 ALoss: 0.048859234899282455 CLoss: 0.01709098182618618\n",
      "Process: 4 Maxrun: 79 ALoss: 0.055995818227529526 CLoss: 0.01758178137242794\n",
      "Process: 0 Maxrun: 26 ALoss: 0.007917070761322975 CLoss: 0.017373856157064438\n",
      "Process: 1 Maxrun: 92 ALoss: -0.00637167040258646 CLoss: 0.018084077164530754\n",
      "Process: 2 Maxrun: 113 ALoss: 0.013991385698318481 CLoss: 0.017224594950675964\n",
      "Process: 5 Maxrun: 113 ALoss: 0.010692465119063854 CLoss: 0.015049097128212452\n",
      "Process: 3 Maxrun: 237 ALoss: -0.00819871760904789 CLoss: 0.015625208616256714\n",
      "Process: 1 Maxrun: 185 ALoss: -0.0022071192506700754 CLoss: 0.017387650907039642\n",
      "Process: 5 Maxrun: 126 ALoss: 0.006722076330333948 CLoss: 0.016636032611131668\n",
      "Process: 2 Maxrun: 211 ALoss: 0.012815132737159729 CLoss: 0.017724361270666122\n",
      "Process: 6 Maxrun: 293 ALoss: -0.010712338611483574 CLoss: 0.015908921137452126\n",
      "Process: 0 Maxrun: 275 ALoss: 0.006915852427482605 CLoss: 0.016122516244649887\n",
      "Process: 4 Maxrun: 305 ALoss: 0.016560083255171776 CLoss: 0.017924876883625984\n",
      "Process: 3 Maxrun: 274 ALoss: -0.018615158274769783 CLoss: 0.015086817555129528\n",
      "Process: 0 Maxrun: 201 ALoss: 0.03491530939936638 CLoss: 0.017770064994692802\n",
      "Process: 2 Maxrun: 327 ALoss: 0.009888960979878902 CLoss: 0.017793109640479088\n",
      "Process: 1 Maxrun: 389 ALoss: 0.021288828924298286 CLoss: 0.017041906714439392\n",
      "Process: 6 Maxrun: 348 ALoss: -0.03567800298333168 CLoss: 0.017871439456939697\n",
      "Process: 5 Maxrun: 425 ALoss: 0.022365232929587364 CLoss: 0.017463333904743195\n",
      "Process: 4 Maxrun: 482 ALoss: 0.01701737940311432 CLoss: 0.017900114879012108\n",
      "Process: 0 Maxrun: 179 ALoss: 0.021197980269789696 CLoss: 0.01679874025285244\n",
      "Process: 3 Maxrun: 265 ALoss: -0.002953128656372428 CLoss: 0.017196258530020714\n",
      "Process: 1 Maxrun: 168 ALoss: 0.004223119001835585 CLoss: 0.017359929159283638\n",
      "Process: 6 Maxrun: 224 ALoss: 0.03242839500308037 CLoss: 0.018847836181521416\n",
      "Process: 5 Maxrun: 153 ALoss: -0.002570148790255189 CLoss: 0.017431622371077538\n",
      "Process: 2 Maxrun: 276 ALoss: 0.009881271980702877 CLoss: 0.01740526221692562\n",
      "Process: 4 Maxrun: 81 ALoss: -0.6857172250747681 CLoss: 0.9693224430084229\n",
      "Process: 3 Maxrun: 267 ALoss: 0.0038571960758417845 CLoss: 0.01785328797996044\n",
      "Process: 0 Maxrun: 315 ALoss: -0.006190071813762188 CLoss: 0.017082763835787773\n",
      "Process: 6 Maxrun: 191 ALoss: 0.0013697510585188866 CLoss: 0.015550321899354458\n",
      "Process: 2 Maxrun: 190 ALoss: 0.01048066932708025 CLoss: 0.01653381623327732\n",
      "Process: 4 Maxrun: 241 ALoss: 0.013523707166314125 CLoss: 0.015993107110261917\n",
      "Process: 1 Maxrun: 375 ALoss: 0.016562793403863907 CLoss: 0.017268268391489983\n",
      "Process: 5 Maxrun: 342 ALoss: -0.0030456369277089834 CLoss: 0.01661066897213459\n",
      "Process: 0 Maxrun: 164 ALoss: 0.01629476062953472 CLoss: 0.01565011404454708\n",
      "Process: 6 Maxrun: 188 ALoss: 0.011707878671586514 CLoss: 0.016592398285865784\n",
      "Process: 3 Maxrun: 345 ALoss: 0.013917666859924793 CLoss: 0.018654130399227142\n",
      "Process: 1 Maxrun: 240 ALoss: 0.012477785348892212 CLoss: 0.01699119620025158\n",
      "Process: 2 Maxrun: 332 ALoss: 0.006511287298053503 CLoss: 0.016293291002511978\n",
      "Process: 4 Maxrun: 287 ALoss: -0.000990531058050692 CLoss: 0.016231227666139603\n",
      "Process: 5 Maxrun: 357 ALoss: 0.02647736482322216 CLoss: 0.016923487186431885\n",
      "Process: 0 Maxrun: 220 ALoss: 0.025174234062433243 CLoss: 0.016804134473204613\n",
      "Process: 2 Maxrun: 289 ALoss: -0.00027518050046637654 CLoss: 0.016064565628767014\n",
      "Process: 6 Maxrun: 482 ALoss: -0.009736618958413601 CLoss: 0.017442356795072556\n",
      "Process: 1 Maxrun: 364 ALoss: -0.03220698982477188 CLoss: 0.017145156860351562\n",
      "Process: 3 Maxrun: 441 ALoss: 0.020765474066138268 CLoss: 0.015180799178779125\n",
      "Process: 4 Maxrun: 487 ALoss: -0.0010205254657194018 CLoss: 0.01666555553674698\n",
      "Process: 5 Maxrun: 374 ALoss: 0.024551043286919594 CLoss: 0.015927141532301903\n",
      "Process: 0 Maxrun: 386 ALoss: -0.058771830052137375 CLoss: 0.01872127316892147\n",
      "Process: 2 Maxrun: 163 ALoss: 0.004682890605181456 CLoss: 0.016669180244207382\n",
      "Process: 6 Maxrun: 218 ALoss: 0.003307742765173316 CLoss: 0.016937075182795525\n",
      "Process: 1 Maxrun: 201 ALoss: -0.029497995972633362 CLoss: 0.01723523810505867\n",
      "Process: 3 Maxrun: 344 ALoss: -0.036266181617975235 CLoss: 0.015785323455929756\n",
      "Process: 5 Maxrun: 315 ALoss: 0.006983706261962652 CLoss: 0.016131630167365074\n",
      "Process: 4 Maxrun: 391 ALoss: 0.007171063218265772 CLoss: 0.016697829589247704\n",
      "Process: 2 Maxrun: 148 ALoss: 0.005116026382893324 CLoss: 0.016602303832769394\n",
      "Process: 0 Maxrun: 313 ALoss: 0.007027934771031141 CLoss: 0.016223153099417686\n",
      "Process: 6 Maxrun: 312 ALoss: 0.0076750158332288265 CLoss: 0.016855677589774132\n",
      "Process: 1 Maxrun: 500 ALoss: 0.008754733018577099 CLoss: 0.01736665703356266\n",
      "Process: 3 Maxrun: 356 ALoss: 0.041376594454050064 CLoss: 0.01726256124675274\n",
      "Process: 4 Maxrun: 238 ALoss: 0.01372232474386692 CLoss: 0.016922984272241592\n",
      "Process: 5 Maxrun: 292 ALoss: 0.02869562804698944 CLoss: 0.018194440752267838\n",
      "Process: 2 Maxrun: 344 ALoss: 0.014138790778815746 CLoss: 0.016688773408532143\n",
      "Process: 0 Maxrun: 283 ALoss: 0.04533027485013008 CLoss: 0.016583533957600594\n",
      "Process: 6 Maxrun: 224 ALoss: -0.0013000544859096408 CLoss: 0.01804490201175213\n",
      "Process: 1 Maxrun: 132 ALoss: 0.008855710737407207 CLoss: 0.01646926999092102\n",
      "Process: 5 Maxrun: 163 ALoss: 0.02114536426961422 CLoss: 0.01668201945722103\n",
      "Process: 3 Maxrun: 500 ALoss: -0.014394471421837807 CLoss: 0.01750846579670906\n",
      "Done 45 seconds\n",
      "Done 45 seconds\n",
      "Done 45 seconds\n",
      "Done 45 seconds\n",
      "Process: 4 Maxrun: 500 ALoss: 0.017487185075879097 CLoss: 0.016969462856650352\n",
      "Done 45 seconds\n",
      "Done 45 seconds\n",
      "Process: 2 Maxrun: 385 ALoss: -0.0030001348350197077 CLoss: 0.013835782185196877\n",
      "Done 45 seconds\n",
      "8513 0\n",
      "CPU times: user 170 ms, sys: 127 ms, total: 297 ms\n",
      "Wall time: 45.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "MasterNode = ActorCritic()\n",
    "MasterNode.share_memory()\n",
    "processes = []\n",
    "#worker_opt = optim.Adam(lr=1e-4,params=MasterNode.parameters())\n",
    "params = {\n",
    "    'epochs':1500,\n",
    "    'n_steps':10,\n",
    "    'n_workers':7,\n",
    "}\n",
    "counter = mp.Value('i',0)\n",
    "eplens = mp.Queue()\n",
    "for i in range(params['n_workers']):\n",
    "    p = mp.Process(target=worker, args=(i,MasterNode,counter,params,eplens))\n",
    "    p.start()\n",
    "    processes.append(p)\n",
    "for p in processes:\n",
    "    p.join()\n",
    "for p in processes:\n",
    "    p.terminate()\n",
    "    \n",
    "print(counter.value,processes[1].exitcode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "eplens_ = []\n",
    "while not eplens.empty():\n",
    "    eplens_.append(eplens.get())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1c2d9ba550>]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAFNCAYAAAAepVXtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xd8XnX5//HXld0madOR7r3ooi0llKlAAWVXUKYgU0QZguMrCAqoKPpDEFkKIhSEAjLLXjJllO5d6J5p05HZ7Pv6/XGfYihJerfNnZM7eT8fjzxyn8/53Odc98lpcvVzPsPcHREREZFEkhR2ACIiIiK7SwmMiIiIJBwlMCIiIpJwlMCIiIhIwlECIyIiIglHCYyIiIgkHCUwIm2Mma00s6NbQBznmdkrTV23pTCzD8zs/Dgd+1dm9rd4HFskUSiBEYkDMzvbzKabWamZbTCzV8zssL04npvZkDrbR5hZJDh+iZktMbMLmib6es//3eBcpWZWXufcpWZWuifHdPfJ7n5cU9fdXWa2NvhMpXW+/hKPc+0JMzvazFbWLXP337r7pSGFJNIiKIERaWJm9hPgL8Dvge5AP+AeYNIeHCulkd3r3T0L6AD8ArjfzEbufsS75u6PuntWcL7jdpy7TtnuxN0SHVf387j7VWEHJCKNUwIj0oTMrCPwG+Ayd3/G3cvcvdrdX3D3nwd1JpjZR2ZWGLTO3GVmaXWO4WZ2mZl9DnxuZu8Fu+YErQNn1D2nRz0HbANGBsc42cwWBOd4x8xGNBBvkpldY2bLzGyLmT1pZp338LOvNbOfm9k8oCwou97MlgetRAvM7OQ69S82s3eC1ynB5/6BmS01s21m9tc9rJtsZn8JPs9yM7vCzHZ7ynEza2dmxWY2vE5Zj6C1pkvw9bKZFQQxvGBmvRs41u/M7KE620PqxhR8vkXBdVpmZhcH5R2BF4B+dVqHutVzvFPq/Lz/Y2b77PRz+YmZzTOzIjObYmbpu3s9RFoaJTAiTetgIAN4tpE6tcDVQNeg/lHAj3aq8y3gQGCku389KBsbtA48UbdikIScAuQA88xsGDAFuArIBV4GXqibJNVxRXCuw4FeRJOgu2P8rPU5k2gLTU6w/RlwKNARuBl4zMy6N/L+44H9gf2Ac6zxvjoN1f0hcDQwBsgDTt2TD+Lu5cBzwFl1is8A3nL3LUR/f95PtIWtP1AN3LEn5wI2AicQbU37PnCnmY1x9yLgJGB1ndahTXXfGCSnjxD9WeYCbwJTzSy1TrXTgWOAQUSv2bl7GKdIi6EERqRpdQE2u3tNQxXcfYa7f+zuNe6+Evg70QSirj+4+9bgj2hDeplZIbAZuAE4192XEP0j+5K7v+Hu1cCtQDvgkHqOcSlwnbuvdfdK4EbgO3vxCOiO4FjlwWd90t03uHvE3R8DVhJNKhryB3cvCq7LO8C4Pah7OnC7u69z963AH2OI+8Wg9WLH147+RI/x5QTm7KAMdy9w92fdvdzdi4k+Mtz55xiToIVuedCa9h/gLeBrMb79TGCqu/8n+HnfQjRhPLBOnb+4e36QeL1I49dVJCEk2nNqkZZuC9DVzFIaSmKCFpLbiP4hb0/03+GMnaqtieFc6929Tz3lvYBVOzbcPWJma4D6Hm/0B541s0idslqifXfWxRDDzr4Ut0VH4VwdnAcgi2jLU0Py67zeHtTf3bq9doojlmt5oru/U0/5m0COme0PFBJ9RPc8gJllEe3r9A3+1+KUHcO5vsLMTgR+BQwl+h/L9sCnMb69vp/3Wr788975Wu3RY0KRlkQtMCJN6yOgkuhjmYbcCywGhrp7B+CXgO1UZ2+WiV/P/xIGzMyAvtSfkKwh2oE1p85XhrvvSfICdeI2s0FEP+sPgS7unkP0c+/8WZvaBqBuYtd3Tw8UJKH/JtoKczbRlo6yYPfPgYHAhODnOLGRQ5URTUp26LHjhZm1A54C/gB0D67T6/zvOu3qXtj5551E9PPv6c9QJCEogRFpQkGfhV8Dd5vZt8ysvZmlmtlxZvanoFo2UAyUBh1EfxjDoTcS7b8QiyeBE8zsqKAfxE+JJlUf1lP3b8DNZtYfwMxyzWy3R0s1IIvoH9+C6KHt+8Dwxt/SJJ4ErjKzXmbWiWiisTceI/pY7ovHR4Fsoq0Z28ysC9Gfe0NmA4ebWV8zywGuqbMvHUgjep1qg9aYo+rs30i0Va+h1p0ngZMtOrQ+lejnLQE+ifUDiiQiJTAiTczd/wz8BLie6B+lNcDlRDuEAvyM6B/DEqKdQJ+o5zA7uxGYHPTPOH0X518CnAPcSbR/zEnASe5eVU/1O4CpwOtmVgJ8zJf7Tuwxd58bxDCNaKvIPjTPH9V7ifaJmUf00dxLQH2fva5X7MvzwPy7zr4PgRqiHWRfr1N+G9G+JluCOo1NtPcq0Y7d84hej6k7drh7IdHHbM8CW4HvEO2nsmP/fOBpYGXw8+9W98DuvgA4L/jcBcCxwMlBfxiRVsvc96alWkSkZTOzk4h2Yh0cdiwi0nTUAiMirYqZZZrZscF8MX2IPtppbFi7iCQgtcCISKsSjA56l+gjqzKij2OucveSUAMTkSalBEZEREQSjh4hiYiISMJRAiMiIiIJJ6Fn4u3atasPGDAg7DBERESkCcyYMWOzu+fGUjehE5gBAwYwffr0sMMQERGRJmBmq3ZdK0qPkERERCThKIERERGRhKMERkRERBKOEhgRERFJOHFLYMwsw8ymmdkcM1tgZjcF5QPN7BMzW2pmT5hZWlCeHmwvDfYPiFdsIiIiktji2QJTCUx097HAOOBYMzsI+CNwu7sPAbYBFwX1LwK2BeW3B/VEREREviJuCYxHlQabqcGXAxOBp4LyycC3gteTgm2C/UeZmcUrPhEREUlcce0DY2bJZjYb2AS8ASwDCt29JqiyFugdvO4NrAEI9hcBXeo55iVmNt3MphcUFMQzfBEREWmh4prAuHutu48D+gATgOFNcMz73D3P3fNyc2OarE9ERERamWaZidfdC83sbeBgIMfMUoJWlj7AuqDaOqAvsNbMUoCOwJbmiE9ERKQ1Kq6oZn1hOesLy6mqcY4e0Y2U5NYxADluCYyZ5QLVQfLSDjiGaMfct4HvAI8D5wHPB2+ZGmx/FOz/j7t7vOITERFp6T74fDP3vruU/ft35vBhXRnbJ+eLBKS6NkJBSSWLNhQzf10x89cXsWpLGWWVtZRX11JWWUNlTeRLxztxTE9uP2Mcqa0giYlnC0xPYLKZJRN9VPWku79oZguBx83sd8As4IGg/gPAI2a2FNgKnBnH2ERERFq0heuL+cEj00lJTuKjZVv461uf0yEjhdzsdLaUVVG4vfqLumYwqGsmQ7plkZmeQvu0ZNqnpdA1K41eOe3oldOOj5dv4U+vLqE24txx5n6kpSR2EhO3BMbd5wL71VO+nGh/mJ3LK4DT4hWPiIhIothYXMFFkz8lOyOV5y47lIzUJP67dAvvf15AcUU1XbPS6ZyZRtesdPbpkc3Inh3ITG/8T/r4fp1IT0nmty8upOaxmdx19n6kpyQ30ydqegm9GrWIiEhrU1ZZw4UPfUpxeTX/vvQQenTMAOCEMT05YUzPvTr2RYcNJDXZ+PXzC7js0Zn87Zz9G+wTs72qhlfm5fPc7HV0aJfKj44YzKheHffq/E1JCYyIiEgLEYk4V06ZxeL8Ev5xXh4je3Vo8nN87+ABAPz6+QX86vkF/P6U0dSddm3ttu3c+dZSXpq3gdLKGvp2bkdhWTUvzd3A0SO6ccXEoYztm9Pkce0uJTAiIiItxIMfruStxZu46eRRHLlPt7id53sHD2BDUQX3vrOMPp3acdmRQwB4a9FGfvLkHKpqIpwwpien7d+HCQM7U1xRw+QPV/LAByuYdPd/uf2MsZyyX5+4xRcLJTAiIiItwOcbS/jjq4s5ang3vndw/7if7+ff2If1heX8v9eW0L1DBssLSrnnnWWM7NmBe88ZT/8umV/U7dgulSuPGsqFhw3k0Y9XMXF497jHtytKYEREREJWXRvhJ0/OITMtmT98e1+aYyWdpCTjT98Zw8biCn727zkAnDWhLzecNIqM1Po792alp/CDwwfHPbZYKIEREREJ2V3/Wcq8dUXc+93xdMvOaLbzpqck8/dz87ju2XlMHN6NU8eH+1hodyiBERERCdHsNYXc9fZSTtmvN8ftu3ejjPZEx3ap3HX2+GY/795SAiMiIhKCmtoID/53JX9+YwndstO58eRRYYeUUJTAiIiINLMF64u45ul5zFtXxNEjuvHbb42mY7vUsMNKKEpgREREmkF1bYS3F2/iyelreXvJJjq1T+Pus8dz/L49mqXTbmujBEZERCQOSitrWJJfwpL8EhZuKOLV+flsLq0iNzudS74+iEu/PpiO7dXqsqeUwIiIiDSxe95Zyp9eXfLFdmZaMocN7crpeX05fFhug9P3S+yUwIiIiDShaSu2cutrSzh6RHfOOKAvw3tk0zunHUlJekzUlJTAiIiINJGi8mqufmI2/Tq3544zx+1yhWjZc7qyIiIiTcDdue7ZeWwsruDpHx6i5CXO9BBORESkCTw9cx0vzt3A1ccMaxGrNbd2SmBERET20pL8Em54fj4HDuzMpS1kraDWTgmMiIjIXlhfWM55/5xGVkYKfzlzHMnqrNsslMCIiIjsoaLt1Zz/4DTKKmt46IIJ9OzYLuyQ2gz1MBIREdkDFdW1fP+R6azYXMbkCyYwomeHsENqU5TAiIiI7IFfPjOPaSu2cseZ4zhkSNeww2lz9AhJRERkN729ZBPPzFrHlROHMGlc77DDaZOUwIiIiOyG8qpafvXcfAbnZnLZxCFhh9Nm6RGSiIjIbrjjrc9Zu62cJy45iPSU5LDDabPUAiMiIhKjxfnF/OP95Zye14cDB3UJO5w2TQmMiIhIDCIR59pn5tGhXSrXHjci7HDaPCUwIiIiu7CpuIIrpsxi1upCrj9hBJ0y08IOqc1THxgREZEG1EacRz5ayZ9f/4zK2gg/OWYYp+ynUUctgRIYERGReizJL+Fn/57DvHVFfG1oV34zaTQDu2aGHZYElMCIiIjUURtx7n9/Obe9/hnZGSncedZ+nDimJ2Za46glUQIjIiIC1NRGmLO2kN+/vJgZq7Zx7Kge3HzKaLpkpYcdmtRDCYyIiLRZNbURnpy+lneWbOKj5VsoqaihQ0YKd5w5jpPH9lKrSwumBEZERNqsu95eyl/e/Jy+ndtx4pheHDakK4cN6UrH9qlhhya7oARGRETapKWbSrjn7WWcPLYXfz1rv7DDkd2keWBERKTN2TEpXbu0ZH590siww5E9oARGRETanMc/XcOnK7dx3Qkj6KpOugkpbgmMmfU1s7fNbKGZLTCzHwflN5rZOjObHXwdX+c915rZUjNbYmbfjFdsIiLSdm0qruAPryzi4EFdOG3/PmGHI3sonn1gaoCfuvtMM8sGZpjZG8G+29391rqVzWwkcCYwCugFvGlmw9y9No4xiohIG3PjCwuorInw+1P31SijBBa3Fhh33+DuM4PXJcAioLH5lycBj7t7pbuvAJYCE+IVn4iItD1vLNzIy/PyuXLiEM2qm+CapQ+MmQ0A9gM+CYouN7O5ZvZPM+sUlPUG1tR521oaT3hERERiVlpZw6+fn88+3bO55OuDww5H9lLcExgzywKeBq5y92LgXmAwMA7YAPx5N493iZlNN7PpBQUFTR6viIi0Tre+toT84gp+f+q+pKVoDEuii+tP0MxSiSYvj7r7MwDuvtHda909AtzP/x4TrQP61nl7n6DsS9z9PnfPc/e83NzceIYvIiKtxOw1hUz+aCXnHtSf/ft32mV9afniOQrJgAeARe5+W53ynnWqnQLMD15PBc40s3QzGwgMBabFKz4REWkbqmsjXPP0XLpnZ/Dzb+4TdjjSROI5CulQ4FxgnpnNDsp+CZxlZuMAB1YCPwBw9wVm9iSwkOgIpss0AklERPbWAx+sYHF+CX8/d3+yM7REQGsRtwTG3T8A6huf9nIj77kZuDleMYmISNuydtt27njzc44Z2Z1vjuoRdjjShNSLSUREWq3fvLAQgBu0XECrowRGRERapbcWbeT1hRu58qih9OnUPuxwpIkpgRERkVanvKqWG6YuYGi3LC46bGDY4UgcxLMTr4iISCjuevtz1m4r5/FLDtKcL62UfqoiItKqzFi1jfveW86p43tz0KAuYYcjcaIERkREWo0Vm8u4ePKn9M5px/UnqONua6YERkREWoUtpZWc/+A0zIyHLphA58y0sEOSOFICIyIiCa+8qpaLJk8nv6iC+7+XxwCtNN3qKYEREZGEtrm0kksemc6ctYX85YxxWuuojdAoJBERSVjvflbAT5+cQ3FFNbecui/H7dtz12+SVkEJjIiIJJyqmgh/fHUxD3ywgmHds/jXxRMY3qND2GFJM1ICIyIiCef65+bx5PS1nH/IAK45bjgZqclhhyTNTAmMiIgklCc/XcOT09dyxcQh/PQb+4QdjoREnXhFRCRhLFhfxK+en89hQ7py1dHDwg5HQqQERkREEkJReTU//NdMOrVP444zx5GcZGGHJCHSIyQREWnxNhSVc83T81hfWM4TPziYLlnpYYckIVMCIyIiLdbi/GLue3c5U+esx4EbTx6leV4EUAIjIiItjLvz0bIt/P295bz7WQHt05I59+D+XHjoQPp2bh92eNJCKIEREZEWwd15ad4G/v7ucuatK6JrVho/+8YwzjmoPzntta6RfJkSGBERaRH+/Ppn3PX2UgZ1zeQPp+7LKfv11vwu0iAlMCIiErrHPlnNXW8v5cwD+vL7U/YlSSOMZBc0jFpEREL19uJN/Or5+RyxTy6/+9ZoJS8SEyUwIiISmjlrCrnssZmM6JnN3WePJyVZf5YkNnqEJCIizaa8qpYX565n+sptfLpqK8sLyuid045/nn8Amen6kySxi+luMbNDgAF167v7w3GKSUREWqFIxPnBv2bw3mcF5LRPZf9+nfjO/n04Zb/edMvOCDs8STC7TGDM7BFgMDAbqA2KHVACIyIiMfv7e8t577MCbjhpJOcdPEB9XWSvxNICkweMdHePdzAiItI6zVi1lVtfX8IJ+/bk/EMGYKbkRfZOLL2l5gM94h2IiIi0ToXbq7hyymx65WTwh2/vq+RFmkSDLTBm9gLRR0XZwEIzmwZU7tjv7ifHPzwREUlkkYjzf0/NZVNJBU9deggdMlLDDklaicYeId3abFGIiEir4+7cMHUBry/cyK9OHMnYvjlhhyStSIMJjLu/C2Bmf3T3X9TdZ2Z/BN6Nc2wiIpKg3J2bXljIIx+v4geHD+LCQweEHZK0MrH0gTmmnrLjmjoQERFpHdyd3720iIc+XMlFhw3kmmOHq9+LNLnG+sD8EPgRMMjM5tbZlQ38N96BiYhI4nF3fv/yIh74YAXnHzKA608YoeRF4qKxPjCPAa8AfwCuqVNe4u5b4xqViIgknJraCNc+M49/z1jLeQf354aTRip5kbhprA9MEVBkZpftvM/MUt29Oq6RiYhI6LaUVjJ3bRFz1xbRLi2J7x08gIzU5K/UK6+q5YopM3lz0SauOnooPz5qqJIXiatYJrKbCfQFtgEG5AD5ZrYR+L67z4hjfCIi0szcnWdmruOOtz5n9dbtAJiBO/zr49XcNGkUR+7TDYgOk56/vojfvriQ6au28dtvjebcg/qHGb60EbEkMG8AT7n7awBm9g3g28CDwD3AgfELT0REmtPyglKue3Y+Hy3fwri+OZx7UH/27dOR0b07MndtIdc/N58LHvyUb47qTlZ6Ku9+tonNpVWkpSRx11njOWFMz7A/grQRtqsVAsxsnrvvu1PZXHcfY2az3X1cA+/rS3S9pO5EJ8S7z93vMLPOwBNEF4dcCZzu7tss2tZ4B3A8sB04391nNhZbXl6eT58+PYaPKSIijSmvquXv7y3jnreXkZ6axDXHDeesA/p9Zb2iyppa7n9vOXf+Zynt0pL5+tBcjhyey9eH5tIlKz2k6KW1MLMZ7p4XS91YWmA2mNkvgMeD7TOAjWaWDEQaeV8N8FN3n2lm2cAMM3sDOB94y91vMbNriHYQ/gXRodlDg68DgXtR646ISFy5O1PnrOePryxmfVEFJ43txa9OHNHg6tDpKclcPnEoF39tEKnJSSRrQUYJSSwJzNnADcBzwfZ/g7Jk4PSG3uTuG4ANwesSM1sE9AYmAUcE1SYD7xBNYCYBDweLRn5sZjlm1jM4joiINCF3579Lt3D7m58xY9U2RvXqwO1njOPAQV1ien99HXlFmtMuExh33wxc0cDupbGcxMwGAPsBnwDd6yQl+UQfMUE0uVlT521rgzIlMCIiTaS4opqnZ6zlkY9XsbygjK5Z6fzx2/vynf37qjVFEsouExgzGwb8jGiflS/qu/vEWE5gZlnA08BV7l5cd1idu7uZNd4J56vHuwS4BKBfv36781YRkTbJ3Zmztogpn6zmhbnr2V5Vy7i+Odx2+liO37enWlMkIcXyCOnfwN+AfwC1u3NwM0slmrw86u7PBMUbdzwaMrOewKagfB3R4do79AnKvsTd7wPug2gn3t2JR0SkrXl9QT63v/k5izYU0z4tmZPG9OKcYGSRSCKLJYGpcfd7d/fAwaiiB4BF7n5bnV1TgfOAW4Lvz9cpv9zMHifaebdI/V9ERPbcq/M38KNHZzKkWxY3nzKak8f2IjsjNeywRJpELAnMC2b2I+BZoHJHYQzLCRwKnAvMM7PZQdkviSYuT5rZRcAq/tcR+GWiQ6iXEh1GfUGsH0JERL7sg883c+WU2Yztm8O/LjqQzPRYft2LJI5Y7ujzgu8/r1PmwKDG3uTuHxCdubc+R9VT34GvLFsgIiK7Z+bqbVzyyHQG5Wby0PkTlLxIqxTLKKSBzRGIiIjsvcX5xVzw4KfkZqfz8EUT6Nhej4ykdUraVQUza29m15vZfcH2UDM7Mf6hiYjI7lheUMo5/5hGRmoS/7rowAYnoxNpDXaZwBBd86gKOCTYXgf8Lm4RiYjIblu7bTvn/OMTIu48evGB9O3cPuyQROIqlgejg939DDM7C8Ddt5vWSBcRCcXm0kr++cEKlheUsX//Thw0qAtds9P47j8+obSyhimXHMSQbtlhhykSd7EkMFVm1o5ox13MbDB1RiOJiEj8bSqu4O/vLefRT1ZRWROhV8d2vLog/4v97dOS+dfFBzKql+Z3kbYhlgTmBuBVoK+ZPUp0ePT58QxKRKQtc3fe/ayAaSu2snRTKcsKSlm5ZTsAk8b24kdHDmFItyzyiyr4ZMUW5q0t4vgxPRnfr1PIkYs0H4uOXt5FJbMuwEFEh0V/DKS5+/o4x7ZLeXl5Pn369LDDEBFpMgvXF/PbFxfy0fItpCQZA7pmMjg3k6Hdsjktrw/9u2SGHaJI3JjZDHfPi6VuTJMDuPsW4KU6J1gNaCEiEZEmUFMbYdGGEh6btorHP11DTrtUfjNpFGce0I+0lFjGWoi0PXs6u5E68YqI7IWyyhoe+XgV/126mZmrtlFWVUtKknHhoQO5cuJQzd8isgt7msBoEUURkT3g7ry2YCO/eWEB64sqGN4jm2/v34e8AZ05aGBnunXQ3C0isWgwgTGzO6k/UTEgJ24RiYi0Qu7O4vwSbn1tCW8t3sTwHtncefZ+7N+/c9ihiSSkxlpgGusdq56zIiK7UFMb4ZmZ63h/6WY+WraFzaWVtE9L5rrjR3D+oQNITVb/FpE91WAC4+6TmzMQEZHW5p53lnHbG5/RvUM6XxvalYMHdeGI4bma4l+kCWiJUhGROCjcXsX97y3nmJHdue/c/dEE5iJNS+2XIiJxcP/7yymtquGn3xim5EUkDpTAiIg0sc2llTz435WcOKYXw3t0CDsckVZplwmMmQ0zs7fMbH6wPcbMro9/aCIiielv7yyjorqWq44eGnYoIq1WLC0w9wPXAtUA7j4XODOeQYmIJKqNxRU88vEqTtmvD4Nzs8IOR6TViiWBae/u03Yqq4lHMCIiie7ut5dSG3F+fJRaX0TiKZYEZrOZDSaY1M7MvgNsiGtUIiIJaNWWMqZMW81peX3p16V92OGItGqxDKO+DLgPGG5m64AVwDlxjUpEJAH96dUlpCQlcbX6vojE3S4TGHdfDhxtZplAkruXxD8sEZHEMmPVNl6at4EfHzVU6xmJNIPG1kL6SQPlALj7bXGKSUQkobg7v395EbnZ6Vzy9UFhhyPSJjTWApMdfN8HOACYGmyfBOzcqVdEpM16bUE+M1Zt4w+n7ktmuiY4F2kOja2FdBOAmb0HjN/x6MjMbgReapboRERauKqaCLe8sphh3bM4bf8+YYcj0mbEMgqpO1BVZ7sqKBMRafMe/mglK7ds59rjR5Ci1aVFmk0sbZ0PA9PM7FnAgEnAQ/EMSkQkEcxdW8ifXl3CUcO7ccSw3LDDEWlTYhmFdLOZvQJ8jehcMBe4+6y4RyYi0oIVba/mR4/OJDc7nVtPG6sFG0WaWay9zWqBCNEEJhK/cEREWj5356f/nsPG4gqe/MHBdMpMCzskkTYnlsUcfww8CnQFugH/MrMr4h2YiEhLdd97y3lz0UZ+efwI9uvXKexwRNqkWFpgLgIOdPcyADP7I/ARcGc8AxMRaYlmrd7Gn15bwnGje3D+IQPCDkekzYqly7wRfYS0Q21QJiLSplRU1/Kzf8+hW3Y6f/zOGPV7EQlRLC0wDwKf7DQK6YG4RiUi0gL95c3PWVZQxuQLJ9AhIzXscETatFhGId1mZu8AhwVFGoUkIm3OrNXbuO+9ZZx5QF8O15BpkdDtMoExs8HAAnefaWZHAl8zsxXuXhj/8EREwldRXcvPn5pL9w4Z/PKEEWGHIyLE1gfmaaDWzIYAfwP6Ao/FNSoRkRbkL29+ztJNpdzy7TF6dCTSQsSSwETcvQY4FbjL3X8O9NzVm8zsn2a2yczm1ym70czWmdns4Ov4OvuuNbOlZrbEzL65Jx9GRKSpvftZAX/XoyORFieWBKbazM4Cvge8GJTF8l+Qh4Bj6ym/3d3HBV8vA5jZSOBMYFTwnnvMLDmGc4iIxM01YSbQAAAdcElEQVSGonKufmI2w7plc8NJo8IOR0TqiCWBuQA4GLjZ3VeY2UDgkV29yd3fA7bGGMck4HF3r3T3FcBSYEKM7xURaXLVtRGueGwWldW13HPOeNql6f9UIi3JLhMYd1/o7le6+5Rge4W7/3Evznm5mc0NHjHtmMKyN7CmTp21QZmISChufW0J01dt4/en7svg3KywwxGRnTSYwJjZk8H3eUHCseNrnpnN3cPz3QsMBsYBG4A/7+4BzOwSM5tuZtMLCgr2MAwRkYY9NWMtf39vOecc1I9J4/R/KZGWqLFh1D8Ovp/YVCdz9407XpvZ/fyvT806oqObdugTlNV3jPuA+wDy8vK8qWITEYlEnP/3+hLufWcZBw/qwvUnjAw7JBFpQIMtMO6+Ifi+CqgExgJjgMqgbLeZWd3RS6cAO0YoTQXONLP0oI/NUGDanpxDRGRPlFbWcMkj07n3nWWcfWA/Jl84gYxU9XsRaalimcjuYuDXwH+ILiVwp5n9xt3/uYv3TQGOALqa2VrgBuAIMxsHOLAS+AGAuy8IHlktBGqAy9y9tr7jiog0tTVbt3Px5OksLSjlN5NGce5B/bXOkUgLZ+6NP4UxsyXAIe6+JdjuAnzo7vs0Q3yNysvL8+nTp4cdhogksJmrt3HJw9Opqolwz3f357ChXcMOSaTNMrMZ7p4XS91YFnPcApTU2S4JykREEtoLc9bz03/PoWfHDB6/5ACGdNNoI5FEEUsCs5ToatTPE330MwmYa2Y/gehij3GMT0QkLv75wQp+8+JC8vp34r7v5dE5My3skERkN8SSwCwLvnZ4Pvie3fThiIjE37QVW/ndSwv5xsju3Hn2fqSnqLOuSKLZZQLj7jftXGZmKcH6SCIiCaVwexVXPT6Lvp3bc9sZ45S8iCSoxiay+6DO652XDtAQZxFJOO7O/z01l4LSSu48az+y0mNphBaRlqixpQQy67wevdM+jS8UkYTzr09W8/rCjfzfN4czpk9O2OGIyF5oLIHxBl7Xty0i0qItWF/Eb19cyOHDcrnosIFhhyMie6mx9tMcMzuFaJKTY2anBuUGdIx7ZCIiTSS/qIKLHppOl8w0/nz6WJKS1IgskugaS2DeBU6u8/qkOvvei1tEIiJNqKyyhosmf0pJRTVP/fAQumalhx2SiDSBBhMYd7+gOQMREWlqtRHnx4/PYtGGYh447wBG9OwQdkgi0kTUBV9EWq3fv7yINxdt4reTRnHk8G5hhyMiTaixTrwiIgnr1fkbeOCDFZx/yADOPXhA2OGISBNTAiMirc76wnJ+8fQ8xvbpyHUnjAg7HBGJg5geIZnZIcCAuvXd/eE4xSQissdqI85VT8ympjbCHWfuR2qy/p8m0hrtMoEJZuEdDMwGaoNiB5TAiEiLc/fbS5m2Yit/Pm0sA7pm7voNIpKQYmmByQNGursmrxORFu29zwq4463PmTSuF6eO7x12OCISR7EkMPOBHsCGOMciIrJH1heWc8sri5k6Zz2Dumbyu2+NxkyT1Ym0ZrEkMF2BhWY2DajcUejuJzf8FhGR+KuqiXDPO0v527vLcIcrJw7h0iMG0z5NM0SItHax/Cu/Md5BiIjsrnWF5Vz26ExmrynkxDE9uea44fTp1D7ssESkmewygXH3d5sjEBGRWL29ZBNXPzGb2lrn3u+O57h9e4Ydkog0s1hGIR0E3AmMANKAZKDM3TUnt4g0q9qI85c3P+PO/yxleI9s7j1nfwZqpJFImxTLI6S7gDOBfxMdkfQ9YFg8gxIR2dmm4gqufHwWHy/fymn79+G33xpNRmpy2GGJSEhi6unm7kvNLNnda4EHzWwWcG18QxMRifpw6WaufHw2pZXV3HraWL6zf5+wQxKRkMWSwGw3szRgtpn9iehwak1tKSLNYsq01fzy2XkMzs3ise8fyLDu2WGHJCItQCyJyLlBvcuBMqAv8O14BiUiAvDi3PX88tl5HDEsl+cvO1TJi4h8IZZRSKvMrB3Q091vaoaYRER497MCrn5iNgf078y95+yv/i4i8iW7bIExs5OIroP0arA9zsymxjswEWm7Zq7exqWPzGBIt2zuPy9PyYuIfEWsE9lNAN4BcPfZZjYwjjGJSBvk7sxZW8S/p6/huVnr6N4hnYcvnEDHdqlhhyYiLVAsCUy1uxfttK6IFnYUkSZRXlXL45+u5rFPVvP5plLSU5I4bnQPfn7scHKz08MOT0RaqFgSmAVmdjaQbGZDgSuBD+Mbloi0dsUV1Tzy0Sr++cEKtpRVMa5vDn84dV9OGNOTDhlqdRGRxsWSwFwBXEd0IccpwGvAb+MZlIgkBvdoY2zdFtpIxFm9dTsLNxRTUlHN8B4d2KdHNhmpyZRV1vDeZwW8sWgjbyzYSEllDYcPy+WyI4cwYWDnsD6GiCSgWEYhbSeawFwX/3BEJGyzVm9j7toiMlKTSE9JJiM1mb6d2zE4N4uM1GTcnYUbinlu1jqen72erWVVdGyXSsf2qbRLTWbl5jLKqmq/dMwkg/5dMllXWE5VTYSO7VI5ZlR3Ljx0IKN7dwzpk4pIImswgdnVSCN3P7npwxGRsKwrLOcPLy/ixbkb6t2/IwlJTjKWbiolNdk4Yp9uDOmWRVF5NUXbqymrquGAAZ0Z0TObkT07kp2RwuL8EhZuKGZJfjFH7tONY0Z254ABnUhJ1nyYIrLnGmuBORhYQ/Sx0SeANVJXRBJURXUt97yzjL+/uwyAK48ayjkH9qMm4pRX17K9spZVW8v4bGMpSzeVUFJRw3kH9+fEMb3olJm2y+MP6JrJsaN7xPtjiEgb01gC0wM4BjgLOBt4CZji7guaIzARib8l+SVcOWUWSzaWcNLYXlxz3HB657T7Sr19++gxj4i0LA0mMMHCja8Cr5pZOtFE5h0zu8nd72quAEWk6bk7//p4Fb97aRHZGSk8eMEBHLlPt7DDEhGJWaOdeIPE5QSiycsA4K/As/EPS0TipbSyhp88MZvXF27k8GG53HraWM23IiIJp7FOvA8Do4GXgZvcff7uHNjM/gmcCGxy99FBWWfgCaLJ0ErgdHffZtExmHcAxwPbgfPdfeZufxoRadS6wnIueuhTPt9UyvUnjODCQweSlKTubSKSeBobBnAOMBT4MfChmRUHXyVmVhzDsR8Cjt2p7BrgLXcfCrwVbAMcF5xrKHAJcG/sH0FEYjF7TSGT7vov67aV8+D5B3Dx1wYpeRGRhNVYH5i9GuPo7u+Z2YCdiicBRwSvJxNdX+kXQfnDHp0V62MzyzGznu5e/3hOEdktry3I58ops+jWIZ0p3z+Qod2zww5JRGSvxDITb1PqXicpyQe6B697Ex2yvcPaoEwJjMheemXeBi6fMosxfTryj+/l0SVL/V1EJPGFNpNU0Nqy24tCmtklZjbdzKYXFBTEITKR1uPlIHkZ1zeHRy46UMmLiLQazZ3AbDSzngDB901B+Tqgb516fYKyr3D3+9w9z93zcnNz4xqsSCJ7ed4GrgiSl8kXTiArvbkbXEVE4qe5f6NNBc4Dbgm+P1+n/HIzexw4EChS/xeR3be+sJxX5+fzyvwNTF+1jf37deIhJS8i0grF7beamU0h2mG3q5mtBW4gmrg8aWYXAauA04PqLxMdQr2U6DDqC+IVl0hrNH9dEbe8spgPlm4GYHiPbK4+ehgXHTaQTCUvItIKxe03m7uf1cCuo+qp68Bl8YpFpLVaV1jOra8t4dlZ6+icmcZPjxnG8WN6Mjg3K+zQRETiSv81E0lAG4rKue+95Tz6yWoM+NERg7n0iMF0yEgNOzQRkWahBEYkgazZup173lnGUzPWEHE4Zb/eXH3MsHoXYBQRac2UwIgkiBmrtvLdf3xCJAKn5/Xl0sMH07dz+7DDEhEJhRIYkQSwYnMZF0+eTo8OGTz2/YPopRYXEWnjQpvITkRis6W0kvMfnIaZ8dAFE5S8iIigBEakRauoruXih6eTX1TB/d/LY0DXzLBDEhFpEfQISaQFiESckooaCsuryC+qYO7aImavKWTGqm1sLKng3u+OZ//+ncIOU0SkxVACIxKiJfkl/OTJ2SzaUExkp5XBeue0Y/8BnTh5bC++OapHOAGKiLRQSmBEQuDuPDl9DTdMXUB2Rio/OmIInTLTyGmXSuesNEb16kC37IywwxQRabGUwIg0s8LtVdz0wkKenbWOQ4d04S9n7EdutlaJFhHZHUpgRJrB6i3beX1hPm8u2sinK7cRcefqo4dx+cQhJCdZ2OGJiCQcJTAicfb+5wWc/+Cn1Eacfbpnc+nhgzhh316M7NUh7NBERBKWEhiRONpUUsHVT8xmcG4mD5x3gGbOFRFpIkpgROKkNuJc9fhsSitrmPL9g5S8iIg0ISUwInFyz9tL+XDZFv707TEM7Z4ddjgiIq2KZuIViYNPlm/h9jc/Y9K4XpyW1yfscEREWh0lMCJNrKi8mqufmE2/zu25+ZR9MdMoIxGRpqZHSCJN7KYXFrCxpJKnf3gIWen6JyYiEg9qgRFpQq/Oz+eZmeu47IjBjOubE3Y4IiKtlhIYkSayubSS656dx6heHbh84tCwwxERadXUvi3SBNyda5+ZR0llDVPOGEdaiv5vICIST/otK9IE/vXJat5YuJGffWMYwzRkWkQk7pTAiOyldz8r4MapCzhin1wuOmxQ2OGIiLQJSmBE9sLi/GIue3QmQ7tlcdfZ47Uwo4hIM1ECI7KHNhVXcOGDn5KZnsyDFxygIdMiIs1ICYzIHpi3tojzHvyUwvJqHjjvAHp2bBd2SCIibYr+yyiyG1ZsLuPW15fw0twN5LRP5e7vjmd0745hhyUi0uYogRHZhZraCP9dtoVnZ67lxbkbSE1O4oqJQ/j+1wfRISM17PBERNokJTAi9XB35q4tYuqc9Uyds56Ckko6ZKRwzkH9+dGRg+mWnRF2iCIibZoSGJE6Fq4vZuqc9bw0bz1rtpaTmmwcuU83Th3fmyOHdyM9JTnsEEVEBCUwIpRX1fLi3PU8+slqZq8pJCXJOHRIV66YOJRvjuxBx/Z6TCQi0tIogZE2xd2ZuXobi/NLWFFQxsotZUxbsZXiihoG52by6xNHcsp+vemUmRZ2qCIi0gglMNJmzFi1ld+/vJgZq7YBkJ6SxIAumRw1ojun5/XloEGdMdNEdCIiiUAJjLR6ywtKueWVxby+cCPdstO5+ZTRHLlPN3p0yCBJM+eKiCQkJTDSqj0/ex3XPD2P5CTjZ98YxoWHDaR9mm57EZFEp9/k0ipV1US4+aWFTP5oFQcM6MRdZ4+newcNfRYRaS2UwEirs2B9Edc/N59Zqwu5+LCB/OK44aQma9UMEZHWJJQExsxWAiVALVDj7nlm1hl4AhgArAROd/dtYcQniaemNsJrCzYy+cOVTFu5laz0FO4+ezwnjOkZdmgiIhIHYbbAHOnum+tsXwO85e63mNk1wfYvwglNEkVReTWPT1vNQx+uZENRBX07t+O640dwel5fzd8iItKKtaRHSJOAI4LXk4F3UAIjDVhXWM4D76/giU9XU1ZVy8GDunDTyaM4akR3kjWySESk1QsrgXHgdTNz4O/ufh/Q3d03BPvzge71vdHMLgEuAejXr19zxCotyMbiCu5+eylTpq3GHU4c05OLvzZIK0KLiLQxYSUwh7n7OjPrBrxhZovr7nR3D5KbrwiSnfsA8vLy6q0jrc+2sirufnspj3y8itqIc1peXy6fOITeOe3CDk1EREIQSgLj7uuC75vM7FlgArDRzHq6+wYz6wlsCiM2aVmqaiI8/NFK/vrW55RW1nDq+D5cOXEo/bq0Dzs0EREJUbMnMGaWCSS5e0nw+hvAb4CpwHnALcH355s7Nmk5aiPOawvy+dOri1m5ZTtfH5bL9SeMYFj37LBDExGRFiCMFpjuwLPBmjMpwGPu/qqZfQo8aWYXAauA00OITUJWXlXLUzPX8sD7y1m5ZTtDu2Xx0AUHcMQ+3cIOTUREWpBmT2DcfTkwtp7yLcBRzR2PhC8ScWat2caLczfw3Kx1bNtezdi+Odz9zeF8c1R3UjQJnYiI7KQlDaOWNqS8qpZPVmzhnSUFvDo/n/ziCtKSk5g4vBsXfW0gef07aWVoERFpkBIYaTYbiyt4Yc56/rN4E9NXbqOqNkJ6ShJfH5bLNfsO56gR3cjO0ORzIiKya0pgJK5KKqp5bcFGnpu1jg+XbSbiMLxHNucfOoCvDe3KAQM6k5GaHHaYIiKSYJTASJOrqonw3mcFPDd7HW8s3EhlTYR+ndtz+cShTBrXi8G5WWGHKCIiCU4JjOyRssoaHEhJMlKTk1hfWM77n2/m/c8L+O/SzRRX1NA5M40zDujLpHG9Gd8vR31aRESkySiBkZgVlFTy8rwNTJ2znhmr6l8ovFfHDI4d3YNjR/fga0NzSdUIIhERiQMlMG3A5tJKZq7axqw1hawoKKNju1S6ZKXRJSudzLRkkpKMpKB1pKCkkg1F5awvLGdLWRWRiFPrTk2t89nGki/6sPz4qKFkpadQVRuhptbJaZ/KYUO7MqhrplpaREQk7pTAtHC1EaekopqSihoqa2qpqI5QUV3L5tJK1hdWsL6wnI0llVTXRKiJOLWRCJU1EcoqayitrKGkooZNJZVA9HFPvy7tKa2oYWtZFTWR+peS6pCRQq+cdnTNSic5yUhOMpIMjh7RnZPH9dJsuCIiEjolMCErrawhv6iC/KIKVm/dzrKCUpYVlLJicxlby6ooqahp9P0ZqUl075BBRkoyyUlGSnK0T0rH9mn07tSOzLQUhnTLYnz/Tuzbu+MXI34iEae4oprtVbXURpyIO+7QNTudrHTdFiIi0rLpL1Uzc3c+XbmNx6et5o1FG7+SoGSkJjGoaxaje3ckNyudDu1S6dgulez0FDLSkslISSIjNZnOmWn0ymlHp/ape/TIJinJyGmfRo7WRBQRkQSkBCaOaiPOqi1l5BdXsLG4gtVbypk6Zx3LCsrISk/huNE9GNwtix4dMujeIYM+ndrRO6cdSUnqQyIiItIYJTBxUF5Vy5PT13D/+8tZu638S/vG9c3hT98Zw4ljetI+TZdfRERkT+gvaBOJRJxF+cW8vmAjj3y8iq1lVYzvl8MVE4fQt1N7unfMoEeHDDLVv0RERGSv6a/pHqiNOOsLy1m6qZSlm0qZsWobH6/YQuH2agCOGt6NS48YzAEDOoccqYiISOukBCYGtRFnwfqiL2aanbW6kMqayBf7e+e045gR3TlkSBcOHtSVHh0zQoxWRESk9VMCU495a4uYu66QJfklLM4vYdGG4i9GC43q1YFzDurPsO5ZDM7NYlBuFp0z00KOWEREpG1RAlOPP722mPc/30xWegr79MjmxDG9OGhQZw4d0pWuWelhhyciItLmKYGpxw0njSQjNZneOe00Lb6IiEgLpASmHkO6aap8ERGRlkxLBYuIiEjCUQIjIiIiCUcJjIiIiCQcJTAiIiKScJTAiIiISMJRAiMiIiIJRwmMiIiIJBwlMCIiIpJwlMCIiIhIwlECIyIiIgnH3D3sGPaYmRUAq+J0+K7A5jgduzXQ9dk1XaPG6frsmq5R43R9di3RrlF/d8+NpWJCJzDxZGbT3T0v7DhaKl2fXdM1apyuz67pGjVO12fXWvM10iMkERERSThKYERERCThKIFp2H1hB9DC6frsmq5R43R9dk3XqHG6PrvWaq+R+sCIiIhIwlELjIiIiCQcJTD1MLNjzWyJmS01s2vCjidsZtbXzN42s4VmtsDMfhyUdzazN8zs8+B7p7BjDZOZJZvZLDN7MdgeaGafBPfRE2aWFnaMYTKzHDN7yswWm9kiMztY99D/mNnVwb+v+WY2xcwy2vo9ZGb/NLNNZja/Tlm994xF/TW4VnPNbHx4kTePBq7P/wv+jc01s2fNLKfOvmuD67PEzL4ZTtRNRwnMTswsGbgbOA4YCZxlZiPDjSp0NcBP3X0kcBBwWXBNrgHecvehwFvBdlv2Y2BRne0/Are7+xBgG3BRKFG1HHcAr7r7cGAs0Wulewgws97AlUCeu48GkoEz0T30EHDsTmUN3TPHAUODr0uAe5spxjA9xFevzxvAaHcfA3wGXAsQ/M4+ExgVvOee4O9dwlIC81UTgKXuvtzdq4DHgUkhxxQqd9/g7jOD1yVE//D0JnpdJgfVJgPfCifC8JlZH+AE4B/BtgETgaeCKm39+nQEvg48AODuVe5eiO6hulKAdmaWArQHNtDG7yF3fw/YulNxQ/fMJOBhj/oYyDGzns0TaTjquz7u/rq71wSbHwN9gteTgMfdvdLdVwBLif69S1hKYL6qN7CmzvbaoEwAMxsA7Ad8AnR39w3Brnyge0hhtQR/Af4PiATbXYDCOr9I2vp9NBAoAB4MHrP9w8wy0T0EgLuvA24FVhNNXIqAGegeqk9D94x+d3/VhcArwetWd32UwEjMzCwLeBq4yt2L6+7z6HC2NjmkzcxOBDa5+4ywY2nBUoDxwL3uvh9Qxk6Pi9r4PdSJ6P+QBwK9gEy++mhAdtKW75ldMbPriD7+fzTsWOJFCcxXrQP61tnuE5S1aWaWSjR5edTdnwmKN+5oog2+bworvpAdCpxsZiuJPnKcSLS/R07wOAB0H60F1rr7J8H2U0QTGt1DUUcDK9y9wN2rgWeI3le6h76qoXtGv7sDZnY+cCLwXf/fXCmt7voogfmqT4GhQe//NKKdnqaGHFOogv4cDwCL3P22OrumAucFr88Dnm/u2FoCd7/W3fu4+wCi98t/3P27wNvAd4Jqbfb6ALh7PrDGzPYJio4CFqJ7aIfVwEFm1j7497bj+uge+qqG7pmpwPeC0UgHAUV1HjW1GWZ2LNHH2Se7+/Y6u6YCZ5pZupkNJNrZeVoYMTYVTWRXDzM7nmifhmTgn+5+c8ghhcrMDgPeB+bxvz4evyTaD+ZJoB/RVcFPd/edO9y1KWZ2BPAzdz/RzAYRbZHpDMwCznH3yjDjC5OZjSPayTkNWA5cQPQ/UbqHADO7CTiDaLP/LOBion0U2uw9ZGZTgCOIrqi8EbgBeI567pkg8buL6KO37cAF7j49jLibSwPX51ogHdgSVPvY3S8N6l9HtF9MDdGuAK/sfMxEogRGREREEo4eIYmIiEjCUQIjIiIiCUcJjIiIiCQcJTAiIiKScJTAiIiISMJRAiMiDTKzLmY2O/jKN7N1dbZjWhnZzB6sM/9LQ3UuM7PvNkG8U4PYlppZUZ1YD4wlDhFJHBpGLSIxMbMbgVJ3v3WnciP6uyRS7xtDYGZHA5e7e5ta/FCkLVELjIjsNjMbYmYLzexRYAHQ08zuM7PpZrbAzH5dp+4HZjbOzFLMrNDMbjGzOWb2kZl1C+r8zsyuqlP/FjObZmZLzOyQoDzTzJ4OzvtUcK5xuxHzznHcFsT6WtBC866ZLQ8msiSod1sQx1wzu7gpr6GI7B0lMCKyp4YDt7v7yGA15WvcPQ8YCxxjZiPreU9H4F13Hwt8RHRW0PqYu08Afg7sSIauAPLdfSTwW6Krou+pjsAr7j4KqAJuJDp9/2nAb4I6lxBdpHMCcABwmZn124tzikgTUgIjIntq2U5TtZ9lZjOBmcAIoL4EprzO9OUzgAENHPuZeuocRnRafdx9DtGWnz1V7u5vBK/nAe+4e03wesf5vgFcYGaziS6bkUN0/RgRaQFSdl1FRKReZTtemNlQ4MfABHcvNLN/ARn1vKeqzutaGv4dVBlDnb1RN45InfNF6pzPgB+5+1txOL+I7CW1wIhIU+gAlADFZtYT+Ob/b++OUSIIoiCA1odNFryGGHmWzbyC4HWMTL2FguGmYjR4BjMzszGYNjNQ2A4++94FKi2qu+kJGcckN0lSVdf5feE5packd1W1G5lXVbWfnAn8kQUGOIXXJEuS92w/BB8nZNwneayqZWQtST4n5Px4yPbj8dv20CofSQ4T84B/8IwaaGEsIbt1Xb/GkdVzkstxdwU4MxYYoIuLJC+jyFSSW+UFzpcFBgBoxyVeAKAdBQYAaEeBAQDaUWAAgHYUGACgHQUGAGjnG9EY7EwRBzdRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(9,5))\n",
    "x = np.array(eplens_)\n",
    "N = 50\n",
    "x = np.convolve(x, np.ones((N,))/N, mode='valid')\n",
    "plt.ylabel(\"Mean Episode Length\")\n",
    "plt.xlabel(\"Training Time\")\n",
    "plt.title(\"CartPole Training Evaluation\")\n",
    "plt.plot(x)\n",
    "#plt.savefig(\"avg_rewards_Nstep.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maxrun: 383.68\n"
     ]
    }
   ],
   "source": [
    "steps = 2000\n",
    "env = gym.make(\"CartPole-v1\")\n",
    "env.reset()\n",
    "maxrun = 0\n",
    "state = torch.from_numpy(env.env.state).float()\n",
    "done = False\n",
    "avg_run = 0\n",
    "runs = int(25)\n",
    "for i in range(runs):\n",
    "    maxrun = 0\n",
    "    done = False\n",
    "    env.reset()\n",
    "    state = torch.from_numpy(env.env.state).float()\n",
    "    while(done==False):\n",
    "        #env.render('human')\n",
    "        policy, value = MasterNode(state)\n",
    "        #sample action\n",
    "        action = torch.distributions.Categorical(logits=policy.view(-1)).sample().detach().numpy()\n",
    "        state_, reward, done, lives = env.step(action)\n",
    "        \n",
    "        #print(value,reward)\n",
    "        state = torch.from_numpy(state_).float()\n",
    "        maxrun += 1\n",
    "    avg_run += maxrun\n",
    "avg_run = avg_run / runs\n",
    "env.close()\n",
    "print(\"Maxrun: {}\".format(avg_run,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'TestModel = ActorCritic()\\nenv = gym.make(\"CartPole-v1\")\\nenv.reset()\\nmaxrun = 0\\nstate = torch.from_numpy(env.env.state).float()\\ndone = False\\navg_run = 0\\nruns = int(200)\\nfor i in range(runs):\\n    maxrun = 0\\n    done = False\\n    env.reset()\\n    state = torch.from_numpy(env.env.state).float()\\n    while(done==False):\\n        #env.render(\\'human\\')\\n        policy, value = TestModel(state)\\n        #sample action\\n        action = torch.distributions.Categorical(logits=policy.view(-1)).sample()\\n        state_, reward, done, lives = env.step(env.action_space.sample())\\n        state = torch.from_numpy(state_).float()\\n        maxrun += 1\\n    avg_run += maxrun\\navg_run /= runs\\nenv.close()\\nprint(\"Maxrun: {}\".format(avg_run,))'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''TestModel = ActorCritic()\n",
    "env = gym.make(\"CartPole-v1\")\n",
    "env.reset()\n",
    "maxrun = 0\n",
    "state = torch.from_numpy(env.env.state).float()\n",
    "done = False\n",
    "avg_run = 0\n",
    "runs = int(200)\n",
    "for i in range(runs):\n",
    "    maxrun = 0\n",
    "    done = False\n",
    "    env.reset()\n",
    "    state = torch.from_numpy(env.env.state).float()\n",
    "    while(done==False):\n",
    "        #env.render('human')\n",
    "        policy, value = TestModel(state)\n",
    "        #sample action\n",
    "        action = torch.distributions.Categorical(logits=policy.view(-1)).sample()\n",
    "        state_, reward, done, lives = env.step(env.action_space.sample())\n",
    "        state = torch.from_numpy(state_).float()\n",
    "        maxrun += 1\n",
    "    avg_run += maxrun\n",
    "avg_run /= runs\n",
    "env.close()\n",
    "print(\"Maxrun: {}\".format(avg_run,))'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demonstrating how bootstrapping reduces variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No bootstrapping\n",
      "0.010000000000000009 1.99\n",
      "With bootstrapping\n",
      "0.9901 2.9701\n"
     ]
    }
   ],
   "source": [
    "r1 = [1,1,-1]\n",
    "r2 = [1,1,1]\n",
    "R1,R2 = 0.0,0.0\n",
    "#No bootstrapping\n",
    "for i in range(len(r1)-1,0,-1):\n",
    "    R1 = r1[i] + 0.99*R1\n",
    "for i in range(len(r2)-1,0,-1):\n",
    "    R2 = r2[i] + 0.99*R2\n",
    "print(\"No bootstrapping\")\n",
    "print(R1,R2)\n",
    "#With bootstrapping\n",
    "R1,R2 = 1.0,1.0\n",
    "for i in range(len(r1)-1,0,-1):\n",
    "    R1 = r1[i] + 0.99*R1\n",
    "for i in range(len(r2)-1,0,-1):\n",
    "    R2 = r2[i] + 0.99*R2\n",
    "print(\"With bootstrapping\")\n",
    "print(R1,R2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeprl]",
   "language": "python",
   "name": "conda-env-deeprl-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
